{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/kaggle/input/ieee-fraud-detection/train_identity.csv\n",
      "/kaggle/input/ieee-fraud-detection/train_transaction.csv\n",
      "/kaggle/input/ieee-fraud-detection/test_transaction.csv\n",
      "/kaggle/input/ieee-fraud-detection/test_identity.csv\n",
      "/kaggle/input/ieee-fraud-detection/sample_submission.csv\n",
      "/kaggle/input/ieee-cis-12nd-solution-part-1/X_test2.pkl\n",
      "/kaggle/input/ieee-cis-12nd-solution-part-1/X_train2.pkl\n",
      "/kaggle/input/ieee-cis-12nd-solution-part-1/custom.css\n",
      "/kaggle/input/ieee-cis-12nd-solution-part-1/__results__.html\n",
      "/kaggle/input/ieee-cis-12nd-solution-part-1/__output__.json\n",
      "/kaggle/input/ieee-cis-12nd-solution-part-1/__notebook__.ipynb\n",
      "/kaggle/input/ieee-cis-12nd-solution-part-1/y_train2.pkl\n"
     ]
    }
   ],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load in \n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the \"../input/\" directory.\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
    "\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))\n",
    "\n",
    "# Any results you write to the current directory are saved as output."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt \n",
    "from sklearn import preprocessing\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "import catboost as cb\n",
    "import lightgbm as lgb\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import GridSearchCV, StratifiedKFold,TimeSeriesSplit,KFold,GroupKFold\n",
    "from sklearn.metrics import roc_auc_score\n",
    "import sqlite3\n",
    "import xgboost as xgb\n",
    "import datetime\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from scipy.stats import pearsonr\n",
    "import gc\n",
    "from sklearn.model_selection import TimeSeriesSplit\n",
    "import hashlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "__print__ = print\n",
    "def print(string):\n",
    "    __print__(string)\n",
    "    os.system(f'echo \\\"{string}\\\"')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = pd.read_pickle('../input/ieee-cis-12nd-solution-part-1/X_train2.pkl')\n",
    "X_test = pd.read_pickle('../input/ieee-cis-12nd-solution-part-1/X_test2.pkl')\n",
    "y_train = pd.read_pickle('../input/ieee-cis-12nd-solution-part-1/y_train2.pkl')\n",
    "y_train = y_train.isFraud"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat = ['uid1','id_12', 'id_13', 'id_14', 'id_15', 'id_16', 'id_17', 'id_18', 'id_19', 'id_20', 'id_21', 'id_22', 'id_23', 'id_24', 'id_25', 'id_26', 'id_27', 'id_28', 'id_29',\n",
    "            'id_30', 'id_31', 'id_32', 'id_33', 'id_34', 'id_35', 'id_36', 'id_37', 'id_38', 'DeviceType', 'ProductCD', 'card4', 'card6', 'M4','P_emaildomain',\n",
    "            'R_emaildomain', 'card1', 'card2', 'card3',  'card5', 'addr1', 'addr2', 'M1', 'M2', 'M3', 'M5', 'M6', 'M7', 'M8', 'M9','hour','dow','device_name', 'OS_id_30',  'browser_id_31','ProductID','DeviceInfo__P_emaildomain', \n",
    "        'card1__card5', \n",
    "        'card2__id_20',\n",
    "        'card5__P_emaildomain', \n",
    "        'addr1__card1',\n",
    "        'addr1__addr2',\n",
    "        'card1__card2',\n",
    "        'card2__addr1',\n",
    "        'card1__P_emaildomain',\n",
    "        'card2__P_emaildomain',\n",
    "        'addr1__P_emaildomain',\n",
    "        'DeviceInfo__id_31',\n",
    "        'DeviceInfo__id_20',\n",
    "        'DeviceType__id_31',\n",
    "        'DeviceType__id_20',\n",
    "        'DeviceType__P_emaildomain',\n",
    "        'card1__M4',\n",
    "        'card2__M4',\n",
    "        'addr1__M4',\n",
    "        'P_emaildomain__M4',\n",
    "       'uid1__ProductID',\n",
    "       'uid1__DeviceInfo']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat = set(cat) & set(X_train.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------\n",
      "card5\n",
      "train:0.7\n",
      "test:0.8155339805825242\n",
      "----------------------------------------\n",
      "addr1__M4\n",
      "train:0.6373983739837399\n",
      "test:0.6962699822380106\n",
      "----------------------------------------\n",
      "M7\n",
      "train:1.0\n",
      "test:1.0\n",
      "----------------------------------------\n",
      "addr1__addr2\n",
      "train:0.5136986301369864\n",
      "test:0.569620253164557\n",
      "----------------------------------------\n",
      "id_33\n",
      "train:0.7279693486590039\n",
      "test:0.4859335038363171\n",
      "----------------------------------------\n",
      "dow\n",
      "train:1.0\n",
      "test:1.0\n",
      "----------------------------------------\n",
      "card1__card2\n",
      "train:0.6867254199944919\n",
      "test:0.700765825897562\n",
      "----------------------------------------\n",
      "addr1__P_emaildomain\n",
      "train:0.7479224376731302\n",
      "test:0.7792207792207793\n",
      "----------------------------------------\n",
      "card5__P_emaildomain\n",
      "train:0.7474226804123711\n",
      "test:0.8245329000812348\n",
      "----------------------------------------\n",
      "addr1\n",
      "train:0.5525525525525525\n",
      "test:0.6279863481228669\n",
      "----------------------------------------\n",
      "P_emaildomain__M4\n",
      "train:0.9444444444444444\n",
      "test:0.966824644549763\n",
      "----------------------------------------\n",
      "DeviceType__P_emaildomain\n",
      "train:0.9943502824858758\n",
      "test:0.9887640449438202\n",
      "----------------------------------------\n",
      "R_emaildomain\n",
      "train:1.0\n",
      "test:1.0\n",
      "----------------------------------------\n",
      "card1__P_emaildomain\n",
      "train:0.5412019042800303\n",
      "test:0.5949936788874842\n",
      "----------------------------------------\n",
      "DeviceInfo__P_emaildomain\n",
      "train:0.40294840294840295\n",
      "test:0.3663859769719883\n",
      "----------------------------------------\n",
      "DeviceInfo__id_31\n",
      "train:0.13367799113737075\n",
      "test:0.1395259202158412\n",
      "----------------------------------------\n",
      "P_emaildomain\n",
      "train:1.0\n",
      "test:0.9836065573770492\n",
      "----------------------------------------\n",
      "id_14\n",
      "train:0.9615384615384616\n",
      "test:0.8928571428571429\n",
      "----------------------------------------\n",
      "hour\n",
      "train:1.0\n",
      "test:1.0\n",
      "----------------------------------------\n",
      "ProductID\n",
      "train:0.2539071666588445\n",
      "test:0.3730777187780153\n",
      "----------------------------------------\n",
      "card1__M4\n",
      "train:0.6411827270793449\n",
      "test:0.6816680989642228\n",
      "----------------------------------------\n",
      "id_19\n",
      "train:0.8738049713193117\n",
      "test:0.9085487077534792\n",
      "----------------------------------------\n",
      "DeviceType__id_31\n",
      "train:0.7254901960784313\n",
      "test:0.6894409937888198\n",
      "----------------------------------------\n",
      "uid1\n",
      "train:0.12073629998472034\n",
      "test:0.13567933094626056\n",
      "----------------------------------------\n",
      "id_13\n",
      "train:0.4909090909090909\n",
      "test:0.9642857142857143\n",
      "----------------------------------------\n",
      "M4\n",
      "train:1.0\n",
      "test:1.0\n",
      "----------------------------------------\n",
      "card2__id_20\n",
      "train:0.6699238396321311\n",
      "test:0.6714676652743771\n",
      "----------------------------------------\n",
      "uid1__DeviceInfo\n",
      "train:0.1028989375475554\n",
      "test:0.1150085488352212\n",
      "----------------------------------------\n",
      "M2\n",
      "train:1.0\n",
      "test:1.0\n",
      "----------------------------------------\n",
      "M6\n",
      "train:1.0\n",
      "test:1.0\n",
      "----------------------------------------\n",
      "uid1__ProductID\n",
      "train:0.04352361041676135\n",
      "test:0.051496664668066586\n",
      "----------------------------------------\n",
      "card2__addr1\n",
      "train:0.7194469805660624\n",
      "test:0.7564454196379594\n",
      "----------------------------------------\n",
      "card2\n",
      "train:0.9920159680638723\n",
      "test:0.9979919678714859\n",
      "----------------------------------------\n",
      "card1__card5\n",
      "train:0.6877796901893287\n",
      "test:0.699922931408954\n",
      "----------------------------------------\n",
      "card6\n",
      "train:0.8\n",
      "test:1.0\n",
      "----------------------------------------\n",
      "M3\n",
      "train:1.0\n",
      "test:1.0\n",
      "----------------------------------------\n",
      "id_18\n",
      "train:0.8947368421052632\n",
      "test:0.9444444444444444\n",
      "----------------------------------------\n",
      "device_name\n",
      "train:0.8333333333333334\n",
      "test:0.8823529411764706\n",
      "----------------------------------------\n",
      "M5\n",
      "train:1.0\n",
      "test:1.0\n",
      "----------------------------------------\n",
      "card2__P_emaildomain\n",
      "train:0.7593962710861202\n",
      "test:0.8160279853712832\n",
      "----------------------------------------\n",
      "DeviceInfo__id_20\n",
      "train:0.3211884623725873\n",
      "test:0.25567544238239104\n",
      "----------------------------------------\n",
      "id_30\n",
      "train:0.9868421052631579\n",
      "test:0.8620689655172413\n",
      "----------------------------------------\n",
      "id_20\n",
      "train:0.7189873417721518\n",
      "test:0.6498855835240275\n",
      "----------------------------------------\n",
      "addr1__card1\n",
      "train:0.5559863911542503\n",
      "test:0.5907604795194173\n",
      "----------------------------------------\n",
      "DeviceType__id_20\n",
      "train:0.7169179229480737\n",
      "test:0.6445783132530121\n",
      "----------------------------------------\n",
      "card1\n",
      "train:0.7161514055928576\n",
      "test:0.7328601630927213\n",
      "----------------------------------------\n",
      "card2__M4\n",
      "train:0.9207073588134627\n",
      "test:0.9516509433962265\n",
      "----------------------------------------\n",
      "id_31\n",
      "train:0.7175572519083969\n",
      "test:0.6911764705882353\n"
     ]
    }
   ],
   "source": [
    "for column in cat:\n",
    "    train_set = set(X_train[column])\n",
    "    test_set = set(X_test[column])\n",
    "    tt = train_set.intersection(test_set)\n",
    "    print('----------------------------------------')\n",
    "    print(column)\n",
    "    print(f'train:{len(tt)/len(train_set)}')\n",
    "    print(f'test:{len(tt)/len(test_set)}')\n",
    "    X_train[column] = X_train[column].map(lambda x: -999 if x not in tt else x)\n",
    "    X_test[column] = X_test[column].map(lambda x: -999 if x not in tt else x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-auc:0.873458\tvalidation_1-auc:0.842863\n",
      "Multiple eval metrics have been passed: 'validation_1-auc' will be used for early stopping.\n",
      "\n",
      "Will train until validation_1-auc hasn't improved in 500 rounds.\n",
      "[30]\tvalidation_0-auc:0.91136\tvalidation_1-auc:0.87368\n",
      "[60]\tvalidation_0-auc:0.917935\tvalidation_1-auc:0.87872\n",
      "[90]\tvalidation_0-auc:0.925352\tvalidation_1-auc:0.882396\n",
      "[120]\tvalidation_0-auc:0.933237\tvalidation_1-auc:0.888097\n",
      "[150]\tvalidation_0-auc:0.940295\tvalidation_1-auc:0.892513\n",
      "[180]\tvalidation_0-auc:0.947577\tvalidation_1-auc:0.897253\n",
      "[210]\tvalidation_0-auc:0.954542\tvalidation_1-auc:0.902287\n",
      "[240]\tvalidation_0-auc:0.962088\tvalidation_1-auc:0.906744\n",
      "[270]\tvalidation_0-auc:0.967492\tvalidation_1-auc:0.910183\n",
      "[300]\tvalidation_0-auc:0.971451\tvalidation_1-auc:0.912727\n",
      "[330]\tvalidation_0-auc:0.974951\tvalidation_1-auc:0.915145\n",
      "[360]\tvalidation_0-auc:0.978312\tvalidation_1-auc:0.917239\n",
      "[390]\tvalidation_0-auc:0.98113\tvalidation_1-auc:0.91931\n",
      "[420]\tvalidation_0-auc:0.98363\tvalidation_1-auc:0.920736\n",
      "[450]\tvalidation_0-auc:0.985688\tvalidation_1-auc:0.922168\n",
      "[480]\tvalidation_0-auc:0.987313\tvalidation_1-auc:0.923615\n",
      "[510]\tvalidation_0-auc:0.988851\tvalidation_1-auc:0.925044\n",
      "[540]\tvalidation_0-auc:0.990051\tvalidation_1-auc:0.926171\n",
      "[570]\tvalidation_0-auc:0.99111\tvalidation_1-auc:0.927364\n",
      "[600]\tvalidation_0-auc:0.991959\tvalidation_1-auc:0.92845\n",
      "[630]\tvalidation_0-auc:0.992719\tvalidation_1-auc:0.929388\n",
      "[660]\tvalidation_0-auc:0.993371\tvalidation_1-auc:0.930467\n",
      "[690]\tvalidation_0-auc:0.993959\tvalidation_1-auc:0.931286\n",
      "[720]\tvalidation_0-auc:0.99443\tvalidation_1-auc:0.93205\n",
      "[750]\tvalidation_0-auc:0.994824\tvalidation_1-auc:0.932907\n",
      "[780]\tvalidation_0-auc:0.995212\tvalidation_1-auc:0.933698\n",
      "[810]\tvalidation_0-auc:0.995555\tvalidation_1-auc:0.934139\n",
      "[840]\tvalidation_0-auc:0.995904\tvalidation_1-auc:0.934583\n",
      "[870]\tvalidation_0-auc:0.996133\tvalidation_1-auc:0.935169\n",
      "[900]\tvalidation_0-auc:0.996411\tvalidation_1-auc:0.935529\n",
      "[930]\tvalidation_0-auc:0.996709\tvalidation_1-auc:0.936008\n",
      "[960]\tvalidation_0-auc:0.996945\tvalidation_1-auc:0.936483\n",
      "[990]\tvalidation_0-auc:0.997177\tvalidation_1-auc:0.936864\n",
      "[1020]\tvalidation_0-auc:0.997359\tvalidation_1-auc:0.937107\n",
      "[1050]\tvalidation_0-auc:0.997523\tvalidation_1-auc:0.937418\n",
      "[1080]\tvalidation_0-auc:0.997679\tvalidation_1-auc:0.93773\n",
      "[1110]\tvalidation_0-auc:0.997849\tvalidation_1-auc:0.937948\n",
      "[1140]\tvalidation_0-auc:0.997988\tvalidation_1-auc:0.938339\n",
      "[1170]\tvalidation_0-auc:0.998119\tvalidation_1-auc:0.938567\n",
      "[1200]\tvalidation_0-auc:0.998245\tvalidation_1-auc:0.938663\n",
      "[1230]\tvalidation_0-auc:0.998365\tvalidation_1-auc:0.93894\n",
      "[1260]\tvalidation_0-auc:0.998463\tvalidation_1-auc:0.939079\n",
      "[1290]\tvalidation_0-auc:0.998555\tvalidation_1-auc:0.939184\n",
      "[1320]\tvalidation_0-auc:0.998646\tvalidation_1-auc:0.939312\n",
      "[1350]\tvalidation_0-auc:0.998742\tvalidation_1-auc:0.93932\n",
      "[1380]\tvalidation_0-auc:0.998823\tvalidation_1-auc:0.939581\n",
      "[1410]\tvalidation_0-auc:0.9989\tvalidation_1-auc:0.939646\n",
      "[1440]\tvalidation_0-auc:0.99897\tvalidation_1-auc:0.939796\n",
      "[1470]\tvalidation_0-auc:0.999042\tvalidation_1-auc:0.939898\n",
      "[1500]\tvalidation_0-auc:0.999099\tvalidation_1-auc:0.939965\n",
      "[1530]\tvalidation_0-auc:0.99915\tvalidation_1-auc:0.94006\n",
      "[1560]\tvalidation_0-auc:0.999203\tvalidation_1-auc:0.940171\n",
      "[1590]\tvalidation_0-auc:0.999265\tvalidation_1-auc:0.940291\n",
      "[1620]\tvalidation_0-auc:0.999312\tvalidation_1-auc:0.940373\n",
      "[1650]\tvalidation_0-auc:0.999359\tvalidation_1-auc:0.940504\n",
      "[1680]\tvalidation_0-auc:0.999405\tvalidation_1-auc:0.940578\n",
      "[1710]\tvalidation_0-auc:0.999438\tvalidation_1-auc:0.94059\n",
      "[1740]\tvalidation_0-auc:0.999473\tvalidation_1-auc:0.940628\n",
      "[1770]\tvalidation_0-auc:0.999504\tvalidation_1-auc:0.940737\n",
      "[1800]\tvalidation_0-auc:0.999533\tvalidation_1-auc:0.9408\n",
      "[1830]\tvalidation_0-auc:0.999564\tvalidation_1-auc:0.940809\n",
      "[1860]\tvalidation_0-auc:0.999586\tvalidation_1-auc:0.940855\n",
      "[1890]\tvalidation_0-auc:0.999613\tvalidation_1-auc:0.940964\n",
      "[1920]\tvalidation_0-auc:0.999641\tvalidation_1-auc:0.941066\n",
      "[1950]\tvalidation_0-auc:0.999667\tvalidation_1-auc:0.94108\n",
      "[1980]\tvalidation_0-auc:0.999689\tvalidation_1-auc:0.941113\n",
      "[2010]\tvalidation_0-auc:0.99971\tvalidation_1-auc:0.94119\n",
      "[2040]\tvalidation_0-auc:0.999733\tvalidation_1-auc:0.941166\n",
      "[2070]\tvalidation_0-auc:0.99975\tvalidation_1-auc:0.941171\n",
      "[2100]\tvalidation_0-auc:0.999766\tvalidation_1-auc:0.941189\n",
      "[2130]\tvalidation_0-auc:0.99978\tvalidation_1-auc:0.941164\n",
      "[2160]\tvalidation_0-auc:0.999797\tvalidation_1-auc:0.94112\n",
      "[2190]\tvalidation_0-auc:0.999809\tvalidation_1-auc:0.941127\n",
      "[2220]\tvalidation_0-auc:0.999824\tvalidation_1-auc:0.941151\n",
      "[2250]\tvalidation_0-auc:0.999835\tvalidation_1-auc:0.941192\n",
      "[2280]\tvalidation_0-auc:0.999846\tvalidation_1-auc:0.941213\n",
      "[2310]\tvalidation_0-auc:0.999857\tvalidation_1-auc:0.941285\n",
      "[2340]\tvalidation_0-auc:0.999866\tvalidation_1-auc:0.941316\n",
      "[2370]\tvalidation_0-auc:0.999874\tvalidation_1-auc:0.941362\n",
      "[2400]\tvalidation_0-auc:0.999883\tvalidation_1-auc:0.941403\n",
      "[2430]\tvalidation_0-auc:0.999892\tvalidation_1-auc:0.94144\n",
      "[2460]\tvalidation_0-auc:0.999897\tvalidation_1-auc:0.941433\n",
      "[2490]\tvalidation_0-auc:0.999905\tvalidation_1-auc:0.941406\n",
      "[2520]\tvalidation_0-auc:0.999913\tvalidation_1-auc:0.941403\n",
      "[2550]\tvalidation_0-auc:0.99992\tvalidation_1-auc:0.941385\n",
      "[2580]\tvalidation_0-auc:0.999925\tvalidation_1-auc:0.941402\n",
      "[2610]\tvalidation_0-auc:0.999931\tvalidation_1-auc:0.941449\n",
      "[2640]\tvalidation_0-auc:0.999935\tvalidation_1-auc:0.941468\n",
      "[2670]\tvalidation_0-auc:0.999939\tvalidation_1-auc:0.941499\n",
      "[2700]\tvalidation_0-auc:0.999944\tvalidation_1-auc:0.941547\n",
      "[2730]\tvalidation_0-auc:0.999947\tvalidation_1-auc:0.941558\n",
      "[2760]\tvalidation_0-auc:0.999951\tvalidation_1-auc:0.941607\n",
      "[2790]\tvalidation_0-auc:0.999955\tvalidation_1-auc:0.941642\n",
      "[2820]\tvalidation_0-auc:0.999958\tvalidation_1-auc:0.941655\n",
      "[2850]\tvalidation_0-auc:0.999961\tvalidation_1-auc:0.941611\n",
      "[2880]\tvalidation_0-auc:0.999964\tvalidation_1-auc:0.941582\n",
      "[2910]\tvalidation_0-auc:0.999967\tvalidation_1-auc:0.941608\n",
      "[2940]\tvalidation_0-auc:0.999969\tvalidation_1-auc:0.941589\n",
      "[2970]\tvalidation_0-auc:0.999972\tvalidation_1-auc:0.941599\n",
      "[3000]\tvalidation_0-auc:0.999974\tvalidation_1-auc:0.94162\n",
      "[3030]\tvalidation_0-auc:0.999976\tvalidation_1-auc:0.941623\n",
      "[3060]\tvalidation_0-auc:0.999978\tvalidation_1-auc:0.941644\n",
      "[3090]\tvalidation_0-auc:0.99998\tvalidation_1-auc:0.941639\n",
      "[3120]\tvalidation_0-auc:0.999982\tvalidation_1-auc:0.941658\n",
      "[3150]\tvalidation_0-auc:0.999983\tvalidation_1-auc:0.941661\n",
      "[3180]\tvalidation_0-auc:0.999985\tvalidation_1-auc:0.94172\n",
      "[3210]\tvalidation_0-auc:0.999986\tvalidation_1-auc:0.941643\n",
      "[3240]\tvalidation_0-auc:0.999988\tvalidation_1-auc:0.941645\n",
      "[3270]\tvalidation_0-auc:0.999989\tvalidation_1-auc:0.941656\n",
      "[3300]\tvalidation_0-auc:0.99999\tvalidation_1-auc:0.94171\n",
      "[3330]\tvalidation_0-auc:0.99999\tvalidation_1-auc:0.94175\n",
      "[3360]\tvalidation_0-auc:0.999991\tvalidation_1-auc:0.941779\n",
      "[3390]\tvalidation_0-auc:0.999992\tvalidation_1-auc:0.941846\n",
      "[3420]\tvalidation_0-auc:0.999993\tvalidation_1-auc:0.941882\n",
      "[3450]\tvalidation_0-auc:0.999993\tvalidation_1-auc:0.941891\n",
      "[3480]\tvalidation_0-auc:0.999994\tvalidation_1-auc:0.941874\n",
      "[3510]\tvalidation_0-auc:0.999994\tvalidation_1-auc:0.941828\n",
      "[3540]\tvalidation_0-auc:0.999995\tvalidation_1-auc:0.941815\n",
      "[3570]\tvalidation_0-auc:0.999995\tvalidation_1-auc:0.941835\n",
      "[3600]\tvalidation_0-auc:0.999996\tvalidation_1-auc:0.941804\n",
      "[3630]\tvalidation_0-auc:0.999996\tvalidation_1-auc:0.941787\n",
      "[3660]\tvalidation_0-auc:0.999996\tvalidation_1-auc:0.941795\n",
      "[3690]\tvalidation_0-auc:0.999997\tvalidation_1-auc:0.941781\n",
      "[3720]\tvalidation_0-auc:0.999997\tvalidation_1-auc:0.941771\n",
      "[3750]\tvalidation_0-auc:0.999997\tvalidation_1-auc:0.941803\n",
      "[3780]\tvalidation_0-auc:0.999997\tvalidation_1-auc:0.941802\n",
      "[3810]\tvalidation_0-auc:0.999998\tvalidation_1-auc:0.941811\n",
      "[3840]\tvalidation_0-auc:0.999998\tvalidation_1-auc:0.941808\n",
      "[3870]\tvalidation_0-auc:0.999998\tvalidation_1-auc:0.941824\n",
      "[3900]\tvalidation_0-auc:0.999998\tvalidation_1-auc:0.941816\n",
      "[3930]\tvalidation_0-auc:0.999998\tvalidation_1-auc:0.94182\n",
      "[3960]\tvalidation_0-auc:0.999998\tvalidation_1-auc:0.941839\n",
      "Stopping. Best iteration:\n",
      "[3475]\tvalidation_0-auc:0.999994\tvalidation_1-auc:0.941907\n",
      "\n",
      "0.9419068979108101\n",
      "[0]\tvalidation_0-auc:0.871519\tvalidation_1-auc:0.844143\n",
      "Multiple eval metrics have been passed: 'validation_1-auc' will be used for early stopping.\n",
      "\n",
      "Will train until validation_1-auc hasn't improved in 500 rounds.\n",
      "[30]\tvalidation_0-auc:0.910197\tvalidation_1-auc:0.892978\n",
      "[60]\tvalidation_0-auc:0.916545\tvalidation_1-auc:0.896897\n",
      "[90]\tvalidation_0-auc:0.923097\tvalidation_1-auc:0.901121\n",
      "[120]\tvalidation_0-auc:0.92805\tvalidation_1-auc:0.904058\n",
      "[150]\tvalidation_0-auc:0.934118\tvalidation_1-auc:0.908125\n",
      "[180]\tvalidation_0-auc:0.939794\tvalidation_1-auc:0.911437\n",
      "[210]\tvalidation_0-auc:0.946567\tvalidation_1-auc:0.915984\n",
      "[240]\tvalidation_0-auc:0.953767\tvalidation_1-auc:0.920784\n",
      "[270]\tvalidation_0-auc:0.961525\tvalidation_1-auc:0.925893\n",
      "[300]\tvalidation_0-auc:0.967104\tvalidation_1-auc:0.93005\n",
      "[330]\tvalidation_0-auc:0.971697\tvalidation_1-auc:0.933478\n",
      "[360]\tvalidation_0-auc:0.975605\tvalidation_1-auc:0.936306\n",
      "[390]\tvalidation_0-auc:0.979\tvalidation_1-auc:0.939165\n",
      "[420]\tvalidation_0-auc:0.981825\tvalidation_1-auc:0.941579\n",
      "[450]\tvalidation_0-auc:0.984095\tvalidation_1-auc:0.943507\n",
      "[480]\tvalidation_0-auc:0.985886\tvalidation_1-auc:0.945249\n",
      "[510]\tvalidation_0-auc:0.987577\tvalidation_1-auc:0.946954\n",
      "[540]\tvalidation_0-auc:0.988988\tvalidation_1-auc:0.948237\n",
      "[570]\tvalidation_0-auc:0.990127\tvalidation_1-auc:0.949417\n",
      "[600]\tvalidation_0-auc:0.991079\tvalidation_1-auc:0.950365\n",
      "[630]\tvalidation_0-auc:0.99189\tvalidation_1-auc:0.951335\n",
      "[660]\tvalidation_0-auc:0.992607\tvalidation_1-auc:0.952122\n",
      "[690]\tvalidation_0-auc:0.993311\tvalidation_1-auc:0.952893\n",
      "[720]\tvalidation_0-auc:0.993876\tvalidation_1-auc:0.953384\n",
      "[750]\tvalidation_0-auc:0.994308\tvalidation_1-auc:0.953742\n",
      "[780]\tvalidation_0-auc:0.994733\tvalidation_1-auc:0.95421\n",
      "[810]\tvalidation_0-auc:0.995058\tvalidation_1-auc:0.954603\n",
      "[840]\tvalidation_0-auc:0.995376\tvalidation_1-auc:0.954921\n",
      "[870]\tvalidation_0-auc:0.995686\tvalidation_1-auc:0.955228\n",
      "[900]\tvalidation_0-auc:0.996017\tvalidation_1-auc:0.955378\n",
      "[930]\tvalidation_0-auc:0.996297\tvalidation_1-auc:0.955591\n",
      "[960]\tvalidation_0-auc:0.996574\tvalidation_1-auc:0.955837\n",
      "[990]\tvalidation_0-auc:0.996831\tvalidation_1-auc:0.956068\n",
      "[1020]\tvalidation_0-auc:0.997052\tvalidation_1-auc:0.956307\n",
      "[1050]\tvalidation_0-auc:0.997205\tvalidation_1-auc:0.956424\n",
      "[1080]\tvalidation_0-auc:0.997376\tvalidation_1-auc:0.956607\n",
      "[1110]\tvalidation_0-auc:0.997572\tvalidation_1-auc:0.956686\n",
      "[1140]\tvalidation_0-auc:0.997725\tvalidation_1-auc:0.956819\n",
      "[1170]\tvalidation_0-auc:0.997874\tvalidation_1-auc:0.956964\n",
      "[1200]\tvalidation_0-auc:0.99801\tvalidation_1-auc:0.957065\n",
      "[1230]\tvalidation_0-auc:0.998137\tvalidation_1-auc:0.957164\n",
      "[1260]\tvalidation_0-auc:0.998257\tvalidation_1-auc:0.957237\n",
      "[1290]\tvalidation_0-auc:0.998388\tvalidation_1-auc:0.957223\n",
      "[1320]\tvalidation_0-auc:0.998475\tvalidation_1-auc:0.957338\n",
      "[1350]\tvalidation_0-auc:0.998568\tvalidation_1-auc:0.957422\n",
      "[1380]\tvalidation_0-auc:0.998652\tvalidation_1-auc:0.957483\n",
      "[1410]\tvalidation_0-auc:0.998734\tvalidation_1-auc:0.957491\n",
      "[1440]\tvalidation_0-auc:0.998823\tvalidation_1-auc:0.957573\n",
      "[1470]\tvalidation_0-auc:0.998905\tvalidation_1-auc:0.957597\n",
      "[1500]\tvalidation_0-auc:0.998972\tvalidation_1-auc:0.957669\n",
      "[1530]\tvalidation_0-auc:0.999045\tvalidation_1-auc:0.957761\n",
      "[1560]\tvalidation_0-auc:0.999098\tvalidation_1-auc:0.957762\n",
      "[1590]\tvalidation_0-auc:0.999156\tvalidation_1-auc:0.957803\n",
      "[1620]\tvalidation_0-auc:0.999217\tvalidation_1-auc:0.957878\n",
      "[1650]\tvalidation_0-auc:0.999272\tvalidation_1-auc:0.957923\n",
      "[1680]\tvalidation_0-auc:0.999324\tvalidation_1-auc:0.957904\n",
      "[1710]\tvalidation_0-auc:0.999368\tvalidation_1-auc:0.957938\n",
      "[1740]\tvalidation_0-auc:0.999412\tvalidation_1-auc:0.957908\n",
      "[1770]\tvalidation_0-auc:0.999449\tvalidation_1-auc:0.957906\n",
      "[1800]\tvalidation_0-auc:0.999489\tvalidation_1-auc:0.957963\n",
      "[1830]\tvalidation_0-auc:0.999523\tvalidation_1-auc:0.95793\n",
      "[1860]\tvalidation_0-auc:0.999555\tvalidation_1-auc:0.95799\n",
      "[1890]\tvalidation_0-auc:0.999596\tvalidation_1-auc:0.958015\n",
      "[1920]\tvalidation_0-auc:0.999626\tvalidation_1-auc:0.958022\n",
      "[1950]\tvalidation_0-auc:0.999647\tvalidation_1-auc:0.958054\n",
      "[1980]\tvalidation_0-auc:0.999671\tvalidation_1-auc:0.958052\n",
      "[2010]\tvalidation_0-auc:0.999691\tvalidation_1-auc:0.958086\n",
      "[2040]\tvalidation_0-auc:0.999711\tvalidation_1-auc:0.958129\n",
      "[2070]\tvalidation_0-auc:0.999731\tvalidation_1-auc:0.958135\n",
      "[2100]\tvalidation_0-auc:0.999751\tvalidation_1-auc:0.958163\n",
      "[2130]\tvalidation_0-auc:0.999767\tvalidation_1-auc:0.958222\n",
      "[2160]\tvalidation_0-auc:0.999779\tvalidation_1-auc:0.958196\n",
      "[2190]\tvalidation_0-auc:0.999793\tvalidation_1-auc:0.95815\n",
      "[2220]\tvalidation_0-auc:0.999806\tvalidation_1-auc:0.958151\n",
      "[2250]\tvalidation_0-auc:0.99982\tvalidation_1-auc:0.95813\n",
      "[2280]\tvalidation_0-auc:0.999832\tvalidation_1-auc:0.95813\n",
      "[2310]\tvalidation_0-auc:0.999843\tvalidation_1-auc:0.958151\n",
      "[2340]\tvalidation_0-auc:0.999854\tvalidation_1-auc:0.958133\n",
      "[2370]\tvalidation_0-auc:0.999864\tvalidation_1-auc:0.958184\n",
      "[2400]\tvalidation_0-auc:0.999872\tvalidation_1-auc:0.958192\n",
      "[2430]\tvalidation_0-auc:0.999881\tvalidation_1-auc:0.958237\n",
      "[2460]\tvalidation_0-auc:0.999888\tvalidation_1-auc:0.958195\n",
      "[2490]\tvalidation_0-auc:0.999896\tvalidation_1-auc:0.958204\n",
      "[2520]\tvalidation_0-auc:0.999904\tvalidation_1-auc:0.958268\n",
      "[2550]\tvalidation_0-auc:0.99991\tvalidation_1-auc:0.958247\n",
      "[2580]\tvalidation_0-auc:0.999917\tvalidation_1-auc:0.958248\n",
      "[2610]\tvalidation_0-auc:0.999924\tvalidation_1-auc:0.958198\n",
      "[2640]\tvalidation_0-auc:0.99993\tvalidation_1-auc:0.958195\n",
      "[2670]\tvalidation_0-auc:0.999935\tvalidation_1-auc:0.958182\n",
      "[2700]\tvalidation_0-auc:0.999939\tvalidation_1-auc:0.958219\n",
      "[2730]\tvalidation_0-auc:0.999943\tvalidation_1-auc:0.958184\n",
      "[2760]\tvalidation_0-auc:0.999947\tvalidation_1-auc:0.958161\n",
      "[2790]\tvalidation_0-auc:0.999952\tvalidation_1-auc:0.958166\n",
      "[2820]\tvalidation_0-auc:0.999955\tvalidation_1-auc:0.958158\n",
      "[2850]\tvalidation_0-auc:0.999959\tvalidation_1-auc:0.958122\n",
      "[2880]\tvalidation_0-auc:0.999962\tvalidation_1-auc:0.958082\n",
      "[2910]\tvalidation_0-auc:0.999965\tvalidation_1-auc:0.958154\n",
      "[2940]\tvalidation_0-auc:0.999968\tvalidation_1-auc:0.958125\n",
      "[2970]\tvalidation_0-auc:0.99997\tvalidation_1-auc:0.958113\n",
      "[3000]\tvalidation_0-auc:0.999973\tvalidation_1-auc:0.958097\n",
      "Stopping. Best iteration:\n",
      "[2527]\tvalidation_0-auc:0.999906\tvalidation_1-auc:0.958283\n",
      "\n",
      "0.9582827867517749\n",
      "[0]\tvalidation_0-auc:0.891178\tvalidation_1-auc:0.862623\n",
      "Multiple eval metrics have been passed: 'validation_1-auc' will be used for early stopping.\n",
      "\n",
      "Will train until validation_1-auc hasn't improved in 500 rounds.\n",
      "[30]\tvalidation_0-auc:0.915614\tvalidation_1-auc:0.89644\n",
      "[60]\tvalidation_0-auc:0.920273\tvalidation_1-auc:0.899038\n",
      "[90]\tvalidation_0-auc:0.926923\tvalidation_1-auc:0.903552\n",
      "[120]\tvalidation_0-auc:0.934768\tvalidation_1-auc:0.908151\n",
      "[150]\tvalidation_0-auc:0.940377\tvalidation_1-auc:0.911702\n",
      "[180]\tvalidation_0-auc:0.945167\tvalidation_1-auc:0.915013\n",
      "[210]\tvalidation_0-auc:0.951331\tvalidation_1-auc:0.919273\n",
      "[240]\tvalidation_0-auc:0.959373\tvalidation_1-auc:0.924889\n",
      "[270]\tvalidation_0-auc:0.96632\tvalidation_1-auc:0.929789\n",
      "[300]\tvalidation_0-auc:0.971046\tvalidation_1-auc:0.933161\n",
      "[330]\tvalidation_0-auc:0.974565\tvalidation_1-auc:0.935974\n",
      "[360]\tvalidation_0-auc:0.978104\tvalidation_1-auc:0.93866\n",
      "[390]\tvalidation_0-auc:0.98094\tvalidation_1-auc:0.940785\n",
      "[420]\tvalidation_0-auc:0.983303\tvalidation_1-auc:0.94228\n",
      "[450]\tvalidation_0-auc:0.985431\tvalidation_1-auc:0.943613\n",
      "[480]\tvalidation_0-auc:0.987065\tvalidation_1-auc:0.944636\n",
      "[510]\tvalidation_0-auc:0.988595\tvalidation_1-auc:0.945653\n",
      "[540]\tvalidation_0-auc:0.989719\tvalidation_1-auc:0.946519\n",
      "[570]\tvalidation_0-auc:0.990771\tvalidation_1-auc:0.947447\n",
      "[600]\tvalidation_0-auc:0.991699\tvalidation_1-auc:0.948162\n",
      "[630]\tvalidation_0-auc:0.992451\tvalidation_1-auc:0.948756\n",
      "[660]\tvalidation_0-auc:0.993067\tvalidation_1-auc:0.949369\n",
      "[690]\tvalidation_0-auc:0.993705\tvalidation_1-auc:0.949836\n",
      "[720]\tvalidation_0-auc:0.994244\tvalidation_1-auc:0.950209\n",
      "[750]\tvalidation_0-auc:0.994687\tvalidation_1-auc:0.950647\n",
      "[780]\tvalidation_0-auc:0.99511\tvalidation_1-auc:0.950998\n",
      "[810]\tvalidation_0-auc:0.995439\tvalidation_1-auc:0.951197\n",
      "[840]\tvalidation_0-auc:0.995781\tvalidation_1-auc:0.951409\n",
      "[870]\tvalidation_0-auc:0.996094\tvalidation_1-auc:0.951641\n",
      "[900]\tvalidation_0-auc:0.996395\tvalidation_1-auc:0.951792\n",
      "[930]\tvalidation_0-auc:0.99663\tvalidation_1-auc:0.951969\n",
      "[960]\tvalidation_0-auc:0.996845\tvalidation_1-auc:0.952158\n",
      "[990]\tvalidation_0-auc:0.997078\tvalidation_1-auc:0.952347\n",
      "[1020]\tvalidation_0-auc:0.997233\tvalidation_1-auc:0.9524\n",
      "[1050]\tvalidation_0-auc:0.997417\tvalidation_1-auc:0.95253\n",
      "[1080]\tvalidation_0-auc:0.997577\tvalidation_1-auc:0.952653\n",
      "[1110]\tvalidation_0-auc:0.99774\tvalidation_1-auc:0.952648\n",
      "[1140]\tvalidation_0-auc:0.997872\tvalidation_1-auc:0.952707\n",
      "[1170]\tvalidation_0-auc:0.998024\tvalidation_1-auc:0.952704\n",
      "[1200]\tvalidation_0-auc:0.998158\tvalidation_1-auc:0.952753\n",
      "[1230]\tvalidation_0-auc:0.998261\tvalidation_1-auc:0.952736\n",
      "[1260]\tvalidation_0-auc:0.998362\tvalidation_1-auc:0.952807\n",
      "[1290]\tvalidation_0-auc:0.99847\tvalidation_1-auc:0.952852\n",
      "[1320]\tvalidation_0-auc:0.998557\tvalidation_1-auc:0.952937\n",
      "[1350]\tvalidation_0-auc:0.998641\tvalidation_1-auc:0.953031\n",
      "[1380]\tvalidation_0-auc:0.998722\tvalidation_1-auc:0.953008\n",
      "[1410]\tvalidation_0-auc:0.9988\tvalidation_1-auc:0.953025\n",
      "[1440]\tvalidation_0-auc:0.99888\tvalidation_1-auc:0.953025\n",
      "[1470]\tvalidation_0-auc:0.998955\tvalidation_1-auc:0.953023\n",
      "[1500]\tvalidation_0-auc:0.999027\tvalidation_1-auc:0.953111\n",
      "[1530]\tvalidation_0-auc:0.999102\tvalidation_1-auc:0.95317\n",
      "[1560]\tvalidation_0-auc:0.999156\tvalidation_1-auc:0.953265\n",
      "[1590]\tvalidation_0-auc:0.999221\tvalidation_1-auc:0.953266\n",
      "[1620]\tvalidation_0-auc:0.999283\tvalidation_1-auc:0.953302\n",
      "[1650]\tvalidation_0-auc:0.999333\tvalidation_1-auc:0.953332\n",
      "[1680]\tvalidation_0-auc:0.999377\tvalidation_1-auc:0.953349\n",
      "[1710]\tvalidation_0-auc:0.999412\tvalidation_1-auc:0.953392\n",
      "[1740]\tvalidation_0-auc:0.999451\tvalidation_1-auc:0.953423\n",
      "[1770]\tvalidation_0-auc:0.999489\tvalidation_1-auc:0.953441\n",
      "[1800]\tvalidation_0-auc:0.999523\tvalidation_1-auc:0.953391\n",
      "[1830]\tvalidation_0-auc:0.999558\tvalidation_1-auc:0.953418\n",
      "[1860]\tvalidation_0-auc:0.999593\tvalidation_1-auc:0.953475\n",
      "[1890]\tvalidation_0-auc:0.999625\tvalidation_1-auc:0.953416\n",
      "[1920]\tvalidation_0-auc:0.999653\tvalidation_1-auc:0.953436\n",
      "[1950]\tvalidation_0-auc:0.999676\tvalidation_1-auc:0.953438\n",
      "[1980]\tvalidation_0-auc:0.999698\tvalidation_1-auc:0.953345\n",
      "[2010]\tvalidation_0-auc:0.999715\tvalidation_1-auc:0.953328\n",
      "[2040]\tvalidation_0-auc:0.999737\tvalidation_1-auc:0.953235\n",
      "[2070]\tvalidation_0-auc:0.999757\tvalidation_1-auc:0.953218\n",
      "[2100]\tvalidation_0-auc:0.999773\tvalidation_1-auc:0.95316\n",
      "[2130]\tvalidation_0-auc:0.999792\tvalidation_1-auc:0.95319\n",
      "[2160]\tvalidation_0-auc:0.99981\tvalidation_1-auc:0.953174\n",
      "[2190]\tvalidation_0-auc:0.999821\tvalidation_1-auc:0.953204\n",
      "[2220]\tvalidation_0-auc:0.999834\tvalidation_1-auc:0.953185\n",
      "[2250]\tvalidation_0-auc:0.999845\tvalidation_1-auc:0.953156\n",
      "[2280]\tvalidation_0-auc:0.999857\tvalidation_1-auc:0.953133\n",
      "[2310]\tvalidation_0-auc:0.999866\tvalidation_1-auc:0.953128\n",
      "[2340]\tvalidation_0-auc:0.999876\tvalidation_1-auc:0.953164\n",
      "Stopping. Best iteration:\n",
      "[1860]\tvalidation_0-auc:0.999593\tvalidation_1-auc:0.953475\n",
      "\n",
      "0.9534749353615616\n",
      "[0]\tvalidation_0-auc:0.873269\tvalidation_1-auc:0.863411\n",
      "Multiple eval metrics have been passed: 'validation_1-auc' will be used for early stopping.\n",
      "\n",
      "Will train until validation_1-auc hasn't improved in 500 rounds.\n",
      "[30]\tvalidation_0-auc:0.907756\tvalidation_1-auc:0.904778\n",
      "[60]\tvalidation_0-auc:0.913228\tvalidation_1-auc:0.909071\n",
      "[90]\tvalidation_0-auc:0.919895\tvalidation_1-auc:0.913443\n",
      "[120]\tvalidation_0-auc:0.925815\tvalidation_1-auc:0.917013\n",
      "[150]\tvalidation_0-auc:0.933268\tvalidation_1-auc:0.921915\n",
      "[180]\tvalidation_0-auc:0.938893\tvalidation_1-auc:0.925772\n",
      "[210]\tvalidation_0-auc:0.944361\tvalidation_1-auc:0.929147\n",
      "[240]\tvalidation_0-auc:0.954802\tvalidation_1-auc:0.936213\n",
      "[270]\tvalidation_0-auc:0.962925\tvalidation_1-auc:0.941245\n",
      "[300]\tvalidation_0-auc:0.968364\tvalidation_1-auc:0.94495\n",
      "[330]\tvalidation_0-auc:0.972586\tvalidation_1-auc:0.947942\n",
      "[360]\tvalidation_0-auc:0.976444\tvalidation_1-auc:0.950623\n",
      "[390]\tvalidation_0-auc:0.979802\tvalidation_1-auc:0.952759\n",
      "[420]\tvalidation_0-auc:0.982426\tvalidation_1-auc:0.954339\n",
      "[450]\tvalidation_0-auc:0.984655\tvalidation_1-auc:0.955869\n",
      "[480]\tvalidation_0-auc:0.986462\tvalidation_1-auc:0.957076\n",
      "[510]\tvalidation_0-auc:0.987997\tvalidation_1-auc:0.958228\n",
      "[540]\tvalidation_0-auc:0.989308\tvalidation_1-auc:0.959276\n",
      "[570]\tvalidation_0-auc:0.990352\tvalidation_1-auc:0.960182\n",
      "[600]\tvalidation_0-auc:0.991207\tvalidation_1-auc:0.96102\n",
      "[630]\tvalidation_0-auc:0.991989\tvalidation_1-auc:0.961782\n",
      "[660]\tvalidation_0-auc:0.992629\tvalidation_1-auc:0.96237\n",
      "[690]\tvalidation_0-auc:0.993261\tvalidation_1-auc:0.962984\n",
      "[720]\tvalidation_0-auc:0.993752\tvalidation_1-auc:0.963478\n",
      "[750]\tvalidation_0-auc:0.994188\tvalidation_1-auc:0.96394\n",
      "[780]\tvalidation_0-auc:0.994568\tvalidation_1-auc:0.96432\n",
      "[810]\tvalidation_0-auc:0.994929\tvalidation_1-auc:0.964653\n",
      "[840]\tvalidation_0-auc:0.995277\tvalidation_1-auc:0.964956\n",
      "[870]\tvalidation_0-auc:0.995631\tvalidation_1-auc:0.965198\n",
      "[900]\tvalidation_0-auc:0.995903\tvalidation_1-auc:0.965414\n",
      "[930]\tvalidation_0-auc:0.996157\tvalidation_1-auc:0.965669\n",
      "[960]\tvalidation_0-auc:0.996399\tvalidation_1-auc:0.965842\n",
      "[990]\tvalidation_0-auc:0.996631\tvalidation_1-auc:0.965992\n",
      "[1020]\tvalidation_0-auc:0.996818\tvalidation_1-auc:0.966145\n",
      "[1050]\tvalidation_0-auc:0.997036\tvalidation_1-auc:0.966301\n",
      "[1080]\tvalidation_0-auc:0.997182\tvalidation_1-auc:0.966442\n",
      "[1110]\tvalidation_0-auc:0.997353\tvalidation_1-auc:0.966594\n",
      "[1140]\tvalidation_0-auc:0.997524\tvalidation_1-auc:0.966703\n",
      "[1170]\tvalidation_0-auc:0.997684\tvalidation_1-auc:0.966826\n",
      "[1200]\tvalidation_0-auc:0.99784\tvalidation_1-auc:0.966962\n",
      "[1230]\tvalidation_0-auc:0.997977\tvalidation_1-auc:0.967066\n",
      "[1260]\tvalidation_0-auc:0.998098\tvalidation_1-auc:0.96717\n",
      "[1290]\tvalidation_0-auc:0.998232\tvalidation_1-auc:0.967292\n",
      "[1320]\tvalidation_0-auc:0.998333\tvalidation_1-auc:0.967356\n",
      "[1350]\tvalidation_0-auc:0.99844\tvalidation_1-auc:0.967424\n",
      "[1380]\tvalidation_0-auc:0.998541\tvalidation_1-auc:0.967516\n",
      "[1410]\tvalidation_0-auc:0.998647\tvalidation_1-auc:0.967638\n",
      "[1440]\tvalidation_0-auc:0.998736\tvalidation_1-auc:0.967718\n",
      "[1470]\tvalidation_0-auc:0.998819\tvalidation_1-auc:0.967813\n",
      "[1500]\tvalidation_0-auc:0.998884\tvalidation_1-auc:0.967849\n",
      "[1530]\tvalidation_0-auc:0.998948\tvalidation_1-auc:0.967866\n",
      "[1560]\tvalidation_0-auc:0.99902\tvalidation_1-auc:0.967895\n",
      "[1590]\tvalidation_0-auc:0.99908\tvalidation_1-auc:0.967914\n",
      "[1620]\tvalidation_0-auc:0.999143\tvalidation_1-auc:0.967918\n",
      "[1650]\tvalidation_0-auc:0.999207\tvalidation_1-auc:0.967982\n",
      "[1680]\tvalidation_0-auc:0.999259\tvalidation_1-auc:0.967988\n",
      "[1710]\tvalidation_0-auc:0.999311\tvalidation_1-auc:0.968012\n",
      "[1740]\tvalidation_0-auc:0.999363\tvalidation_1-auc:0.968118\n",
      "[1770]\tvalidation_0-auc:0.999412\tvalidation_1-auc:0.968157\n",
      "[1800]\tvalidation_0-auc:0.999455\tvalidation_1-auc:0.968165\n",
      "[1830]\tvalidation_0-auc:0.999491\tvalidation_1-auc:0.968148\n",
      "[1860]\tvalidation_0-auc:0.999525\tvalidation_1-auc:0.968169\n",
      "[1890]\tvalidation_0-auc:0.999561\tvalidation_1-auc:0.968185\n",
      "[1920]\tvalidation_0-auc:0.999595\tvalidation_1-auc:0.96823\n",
      "[1950]\tvalidation_0-auc:0.999624\tvalidation_1-auc:0.968209\n",
      "[1980]\tvalidation_0-auc:0.999651\tvalidation_1-auc:0.968243\n",
      "[2010]\tvalidation_0-auc:0.999672\tvalidation_1-auc:0.968255\n",
      "[2040]\tvalidation_0-auc:0.999696\tvalidation_1-auc:0.968231\n",
      "[2070]\tvalidation_0-auc:0.99972\tvalidation_1-auc:0.968246\n",
      "[2100]\tvalidation_0-auc:0.999739\tvalidation_1-auc:0.968241\n",
      "[2130]\tvalidation_0-auc:0.999762\tvalidation_1-auc:0.968267\n",
      "[2160]\tvalidation_0-auc:0.999777\tvalidation_1-auc:0.968246\n",
      "[2190]\tvalidation_0-auc:0.999792\tvalidation_1-auc:0.968261\n",
      "[2220]\tvalidation_0-auc:0.999808\tvalidation_1-auc:0.968266\n",
      "[2250]\tvalidation_0-auc:0.999822\tvalidation_1-auc:0.96829\n",
      "[2280]\tvalidation_0-auc:0.999836\tvalidation_1-auc:0.968275\n",
      "[2310]\tvalidation_0-auc:0.999848\tvalidation_1-auc:0.968272\n",
      "[2340]\tvalidation_0-auc:0.999859\tvalidation_1-auc:0.96824\n",
      "[2370]\tvalidation_0-auc:0.999868\tvalidation_1-auc:0.968214\n",
      "[2400]\tvalidation_0-auc:0.999877\tvalidation_1-auc:0.968191\n",
      "[2430]\tvalidation_0-auc:0.999885\tvalidation_1-auc:0.968146\n",
      "[2460]\tvalidation_0-auc:0.999894\tvalidation_1-auc:0.96812\n",
      "[2490]\tvalidation_0-auc:0.999902\tvalidation_1-auc:0.968112\n",
      "[2520]\tvalidation_0-auc:0.99991\tvalidation_1-auc:0.968095\n",
      "[2550]\tvalidation_0-auc:0.999917\tvalidation_1-auc:0.968103\n",
      "[2580]\tvalidation_0-auc:0.999924\tvalidation_1-auc:0.968094\n",
      "[2610]\tvalidation_0-auc:0.99993\tvalidation_1-auc:0.968119\n",
      "[2640]\tvalidation_0-auc:0.999935\tvalidation_1-auc:0.968108\n",
      "[2670]\tvalidation_0-auc:0.999939\tvalidation_1-auc:0.968127\n",
      "[2700]\tvalidation_0-auc:0.999944\tvalidation_1-auc:0.96816\n",
      "[2730]\tvalidation_0-auc:0.999948\tvalidation_1-auc:0.968116\n",
      "Stopping. Best iteration:\n",
      "[2238]\tvalidation_0-auc:0.999817\tvalidation_1-auc:0.968301\n",
      "\n",
      "0.968300742247599\n",
      "[0]\tvalidation_0-auc:0.87455\tvalidation_1-auc:0.839322\n",
      "Multiple eval metrics have been passed: 'validation_1-auc' will be used for early stopping.\n",
      "\n",
      "Will train until validation_1-auc hasn't improved in 500 rounds.\n",
      "[30]\tvalidation_0-auc:0.911564\tvalidation_1-auc:0.8775\n",
      "[60]\tvalidation_0-auc:0.91808\tvalidation_1-auc:0.8833\n",
      "[90]\tvalidation_0-auc:0.926146\tvalidation_1-auc:0.887359\n",
      "[120]\tvalidation_0-auc:0.932641\tvalidation_1-auc:0.891208\n",
      "[150]\tvalidation_0-auc:0.939363\tvalidation_1-auc:0.895514\n",
      "[180]\tvalidation_0-auc:0.9447\tvalidation_1-auc:0.899261\n",
      "[210]\tvalidation_0-auc:0.951203\tvalidation_1-auc:0.904028\n",
      "[240]\tvalidation_0-auc:0.958634\tvalidation_1-auc:0.908508\n",
      "[270]\tvalidation_0-auc:0.965034\tvalidation_1-auc:0.912907\n",
      "[300]\tvalidation_0-auc:0.970133\tvalidation_1-auc:0.916813\n",
      "[330]\tvalidation_0-auc:0.974113\tvalidation_1-auc:0.920195\n",
      "[360]\tvalidation_0-auc:0.977686\tvalidation_1-auc:0.922965\n",
      "[390]\tvalidation_0-auc:0.980563\tvalidation_1-auc:0.925243\n",
      "[420]\tvalidation_0-auc:0.983214\tvalidation_1-auc:0.927469\n",
      "[450]\tvalidation_0-auc:0.985333\tvalidation_1-auc:0.929026\n",
      "[480]\tvalidation_0-auc:0.987063\tvalidation_1-auc:0.930758\n",
      "[510]\tvalidation_0-auc:0.98859\tvalidation_1-auc:0.932051\n",
      "[540]\tvalidation_0-auc:0.989796\tvalidation_1-auc:0.933325\n",
      "[570]\tvalidation_0-auc:0.990851\tvalidation_1-auc:0.934493\n",
      "[600]\tvalidation_0-auc:0.991643\tvalidation_1-auc:0.935327\n",
      "[630]\tvalidation_0-auc:0.99243\tvalidation_1-auc:0.936303\n",
      "[660]\tvalidation_0-auc:0.993116\tvalidation_1-auc:0.937228\n",
      "[690]\tvalidation_0-auc:0.993618\tvalidation_1-auc:0.938\n",
      "[720]\tvalidation_0-auc:0.994073\tvalidation_1-auc:0.938621\n",
      "[750]\tvalidation_0-auc:0.994495\tvalidation_1-auc:0.93922\n",
      "[780]\tvalidation_0-auc:0.994864\tvalidation_1-auc:0.939579\n",
      "[810]\tvalidation_0-auc:0.99523\tvalidation_1-auc:0.940102\n",
      "[840]\tvalidation_0-auc:0.995577\tvalidation_1-auc:0.940647\n",
      "[870]\tvalidation_0-auc:0.995913\tvalidation_1-auc:0.940919\n",
      "[900]\tvalidation_0-auc:0.996177\tvalidation_1-auc:0.941252\n",
      "[930]\tvalidation_0-auc:0.996415\tvalidation_1-auc:0.941438\n",
      "[960]\tvalidation_0-auc:0.996599\tvalidation_1-auc:0.941682\n",
      "[990]\tvalidation_0-auc:0.996807\tvalidation_1-auc:0.941861\n",
      "[1020]\tvalidation_0-auc:0.997028\tvalidation_1-auc:0.94212\n",
      "[1050]\tvalidation_0-auc:0.997238\tvalidation_1-auc:0.942352\n",
      "[1080]\tvalidation_0-auc:0.997403\tvalidation_1-auc:0.942581\n",
      "[1110]\tvalidation_0-auc:0.997569\tvalidation_1-auc:0.942704\n",
      "[1140]\tvalidation_0-auc:0.997737\tvalidation_1-auc:0.942945\n",
      "[1170]\tvalidation_0-auc:0.997904\tvalidation_1-auc:0.943063\n",
      "[1200]\tvalidation_0-auc:0.998062\tvalidation_1-auc:0.943248\n",
      "[1230]\tvalidation_0-auc:0.99817\tvalidation_1-auc:0.943315\n",
      "[1260]\tvalidation_0-auc:0.998279\tvalidation_1-auc:0.943466\n",
      "[1290]\tvalidation_0-auc:0.998378\tvalidation_1-auc:0.943513\n",
      "[1320]\tvalidation_0-auc:0.998488\tvalidation_1-auc:0.943656\n",
      "[1350]\tvalidation_0-auc:0.998591\tvalidation_1-auc:0.943882\n",
      "[1380]\tvalidation_0-auc:0.998679\tvalidation_1-auc:0.943921\n",
      "[1410]\tvalidation_0-auc:0.99877\tvalidation_1-auc:0.944026\n",
      "[1440]\tvalidation_0-auc:0.99885\tvalidation_1-auc:0.944209\n",
      "[1470]\tvalidation_0-auc:0.998934\tvalidation_1-auc:0.944245\n",
      "[1500]\tvalidation_0-auc:0.998999\tvalidation_1-auc:0.944334\n",
      "[1530]\tvalidation_0-auc:0.999061\tvalidation_1-auc:0.944382\n",
      "[1560]\tvalidation_0-auc:0.999114\tvalidation_1-auc:0.944491\n",
      "[1590]\tvalidation_0-auc:0.999171\tvalidation_1-auc:0.94448\n",
      "[1620]\tvalidation_0-auc:0.999232\tvalidation_1-auc:0.944503\n",
      "[1650]\tvalidation_0-auc:0.999286\tvalidation_1-auc:0.94464\n",
      "[1680]\tvalidation_0-auc:0.999343\tvalidation_1-auc:0.944657\n",
      "[1710]\tvalidation_0-auc:0.999392\tvalidation_1-auc:0.944778\n",
      "[1740]\tvalidation_0-auc:0.999426\tvalidation_1-auc:0.944833\n",
      "[1770]\tvalidation_0-auc:0.999468\tvalidation_1-auc:0.94483\n",
      "[1800]\tvalidation_0-auc:0.999514\tvalidation_1-auc:0.944915\n",
      "[1830]\tvalidation_0-auc:0.999544\tvalidation_1-auc:0.944976\n",
      "[1860]\tvalidation_0-auc:0.999571\tvalidation_1-auc:0.944957\n",
      "[1890]\tvalidation_0-auc:0.999609\tvalidation_1-auc:0.944992\n",
      "[1920]\tvalidation_0-auc:0.999638\tvalidation_1-auc:0.945065\n",
      "[1950]\tvalidation_0-auc:0.999661\tvalidation_1-auc:0.945175\n",
      "[1980]\tvalidation_0-auc:0.999687\tvalidation_1-auc:0.945231\n",
      "[2010]\tvalidation_0-auc:0.999705\tvalidation_1-auc:0.945228\n",
      "[2040]\tvalidation_0-auc:0.999725\tvalidation_1-auc:0.945238\n",
      "[2070]\tvalidation_0-auc:0.999742\tvalidation_1-auc:0.945271\n",
      "[2100]\tvalidation_0-auc:0.999756\tvalidation_1-auc:0.945259\n",
      "[2130]\tvalidation_0-auc:0.999775\tvalidation_1-auc:0.945331\n",
      "[2160]\tvalidation_0-auc:0.999791\tvalidation_1-auc:0.94539\n",
      "[2190]\tvalidation_0-auc:0.999807\tvalidation_1-auc:0.94534\n",
      "[2220]\tvalidation_0-auc:0.999819\tvalidation_1-auc:0.94537\n",
      "[2250]\tvalidation_0-auc:0.999831\tvalidation_1-auc:0.945402\n",
      "[2280]\tvalidation_0-auc:0.999845\tvalidation_1-auc:0.945488\n",
      "[2310]\tvalidation_0-auc:0.999856\tvalidation_1-auc:0.945482\n",
      "[2340]\tvalidation_0-auc:0.999866\tvalidation_1-auc:0.945494\n",
      "[2370]\tvalidation_0-auc:0.999876\tvalidation_1-auc:0.945524\n",
      "[2400]\tvalidation_0-auc:0.999887\tvalidation_1-auc:0.945625\n",
      "[2430]\tvalidation_0-auc:0.999896\tvalidation_1-auc:0.94561\n",
      "[2460]\tvalidation_0-auc:0.999905\tvalidation_1-auc:0.945651\n",
      "[2490]\tvalidation_0-auc:0.999911\tvalidation_1-auc:0.945722\n",
      "[2520]\tvalidation_0-auc:0.999917\tvalidation_1-auc:0.945698\n",
      "[2550]\tvalidation_0-auc:0.999923\tvalidation_1-auc:0.945717\n",
      "[2580]\tvalidation_0-auc:0.999928\tvalidation_1-auc:0.945705\n",
      "[2610]\tvalidation_0-auc:0.999933\tvalidation_1-auc:0.945665\n",
      "[2640]\tvalidation_0-auc:0.999938\tvalidation_1-auc:0.945708\n",
      "[2670]\tvalidation_0-auc:0.999943\tvalidation_1-auc:0.945715\n",
      "[2700]\tvalidation_0-auc:0.999948\tvalidation_1-auc:0.945728\n",
      "[2730]\tvalidation_0-auc:0.999952\tvalidation_1-auc:0.945695\n",
      "[2760]\tvalidation_0-auc:0.999956\tvalidation_1-auc:0.945681\n",
      "[2790]\tvalidation_0-auc:0.999959\tvalidation_1-auc:0.945724\n",
      "[2820]\tvalidation_0-auc:0.999962\tvalidation_1-auc:0.945747\n",
      "[2850]\tvalidation_0-auc:0.999964\tvalidation_1-auc:0.945759\n",
      "[2880]\tvalidation_0-auc:0.999967\tvalidation_1-auc:0.945807\n",
      "[2910]\tvalidation_0-auc:0.999969\tvalidation_1-auc:0.945805\n",
      "[2940]\tvalidation_0-auc:0.999972\tvalidation_1-auc:0.945766\n",
      "[2970]\tvalidation_0-auc:0.999975\tvalidation_1-auc:0.945833\n",
      "[3000]\tvalidation_0-auc:0.999977\tvalidation_1-auc:0.945805\n",
      "[3030]\tvalidation_0-auc:0.999979\tvalidation_1-auc:0.945794\n",
      "[3060]\tvalidation_0-auc:0.999981\tvalidation_1-auc:0.945822\n",
      "[3090]\tvalidation_0-auc:0.999982\tvalidation_1-auc:0.945874\n",
      "[3120]\tvalidation_0-auc:0.999983\tvalidation_1-auc:0.945856\n",
      "[3150]\tvalidation_0-auc:0.999985\tvalidation_1-auc:0.945864\n",
      "[3180]\tvalidation_0-auc:0.999986\tvalidation_1-auc:0.945869\n",
      "[3210]\tvalidation_0-auc:0.999987\tvalidation_1-auc:0.945832\n",
      "[3240]\tvalidation_0-auc:0.999988\tvalidation_1-auc:0.945893\n",
      "[3270]\tvalidation_0-auc:0.999989\tvalidation_1-auc:0.945877\n",
      "[3300]\tvalidation_0-auc:0.99999\tvalidation_1-auc:0.945839\n",
      "[3330]\tvalidation_0-auc:0.999991\tvalidation_1-auc:0.945865\n",
      "[3360]\tvalidation_0-auc:0.999992\tvalidation_1-auc:0.945846\n",
      "[3390]\tvalidation_0-auc:0.999993\tvalidation_1-auc:0.945837\n",
      "[3420]\tvalidation_0-auc:0.999993\tvalidation_1-auc:0.945874\n",
      "[3450]\tvalidation_0-auc:0.999994\tvalidation_1-auc:0.945861\n",
      "[3480]\tvalidation_0-auc:0.999994\tvalidation_1-auc:0.945868\n",
      "[3510]\tvalidation_0-auc:0.999995\tvalidation_1-auc:0.94591\n",
      "[3540]\tvalidation_0-auc:0.999995\tvalidation_1-auc:0.945869\n",
      "[3570]\tvalidation_0-auc:0.999995\tvalidation_1-auc:0.945861\n",
      "[3600]\tvalidation_0-auc:0.999996\tvalidation_1-auc:0.94587\n",
      "[3630]\tvalidation_0-auc:0.999996\tvalidation_1-auc:0.945854\n",
      "[3660]\tvalidation_0-auc:0.999996\tvalidation_1-auc:0.945891\n",
      "[3690]\tvalidation_0-auc:0.999997\tvalidation_1-auc:0.945847\n",
      "[3720]\tvalidation_0-auc:0.999997\tvalidation_1-auc:0.945885\n",
      "[3750]\tvalidation_0-auc:0.999997\tvalidation_1-auc:0.945841\n",
      "[3780]\tvalidation_0-auc:0.999997\tvalidation_1-auc:0.94589\n",
      "[3810]\tvalidation_0-auc:0.999998\tvalidation_1-auc:0.945911\n",
      "[3840]\tvalidation_0-auc:0.999998\tvalidation_1-auc:0.945904\n",
      "[3870]\tvalidation_0-auc:0.999998\tvalidation_1-auc:0.9459\n",
      "[3900]\tvalidation_0-auc:0.999998\tvalidation_1-auc:0.945848\n",
      "[3930]\tvalidation_0-auc:0.999998\tvalidation_1-auc:0.945894\n",
      "[3960]\tvalidation_0-auc:0.999999\tvalidation_1-auc:0.945892\n",
      "[3990]\tvalidation_0-auc:0.999999\tvalidation_1-auc:0.945935\n",
      "[4020]\tvalidation_0-auc:0.999999\tvalidation_1-auc:0.945899\n",
      "[4050]\tvalidation_0-auc:0.999999\tvalidation_1-auc:0.945926\n",
      "[4080]\tvalidation_0-auc:0.999999\tvalidation_1-auc:0.945906\n",
      "[4110]\tvalidation_0-auc:0.999999\tvalidation_1-auc:0.945929\n",
      "[4140]\tvalidation_0-auc:0.999999\tvalidation_1-auc:0.945942\n",
      "[4170]\tvalidation_0-auc:0.999999\tvalidation_1-auc:0.945968\n",
      "[4200]\tvalidation_0-auc:0.999999\tvalidation_1-auc:0.945971\n",
      "[4230]\tvalidation_0-auc:0.999999\tvalidation_1-auc:0.945996\n",
      "[4260]\tvalidation_0-auc:0.999999\tvalidation_1-auc:0.945988\n",
      "[4290]\tvalidation_0-auc:0.999999\tvalidation_1-auc:0.945986\n",
      "[4320]\tvalidation_0-auc:1\tvalidation_1-auc:0.94599\n",
      "[4350]\tvalidation_0-auc:1\tvalidation_1-auc:0.946016\n",
      "[4380]\tvalidation_0-auc:1\tvalidation_1-auc:0.946059\n",
      "[4410]\tvalidation_0-auc:1\tvalidation_1-auc:0.946057\n",
      "[4440]\tvalidation_0-auc:1\tvalidation_1-auc:0.946063\n",
      "[4470]\tvalidation_0-auc:1\tvalidation_1-auc:0.946056\n",
      "[4500]\tvalidation_0-auc:1\tvalidation_1-auc:0.946063\n",
      "[4530]\tvalidation_0-auc:1\tvalidation_1-auc:0.946057\n",
      "[4560]\tvalidation_0-auc:1\tvalidation_1-auc:0.946076\n",
      "[4590]\tvalidation_0-auc:1\tvalidation_1-auc:0.946079\n",
      "[4620]\tvalidation_0-auc:1\tvalidation_1-auc:0.946064\n",
      "[4650]\tvalidation_0-auc:1\tvalidation_1-auc:0.946062\n",
      "[4680]\tvalidation_0-auc:1\tvalidation_1-auc:0.94607\n",
      "[4710]\tvalidation_0-auc:1\tvalidation_1-auc:0.946057\n",
      "[4740]\tvalidation_0-auc:1\tvalidation_1-auc:0.94603\n",
      "[4770]\tvalidation_0-auc:1\tvalidation_1-auc:0.946019\n",
      "[4800]\tvalidation_0-auc:1\tvalidation_1-auc:0.945993\n",
      "[4830]\tvalidation_0-auc:1\tvalidation_1-auc:0.946004\n",
      "[4860]\tvalidation_0-auc:1\tvalidation_1-auc:0.94598\n",
      "[4890]\tvalidation_0-auc:1\tvalidation_1-auc:0.945963\n",
      "Stopping. Best iteration:\n",
      "[4394]\tvalidation_0-auc:1\tvalidation_1-auc:0.946087\n",
      "\n",
      "0.9460870814607542\n",
      "End:0.9536104887464999\n"
     ]
    }
   ],
   "source": [
    "kf=KFold(n_splits = 5)\n",
    "resu1 = 0\n",
    "impor1 = 0\n",
    "y_pred = 0\n",
    "stack_train = np.zeros([X_train.shape[0],])\n",
    "for train_index, test_index in kf.split(X_train, y_train):\n",
    "    X_train2= X_train.iloc[train_index,:]\n",
    "    y_train2= y_train.iloc[train_index]\n",
    "    X_test2= X_train.iloc[test_index,:]\n",
    "    y_test2= y_train.iloc[test_index]\n",
    "    clf = xgb.XGBClassifier(n_estimators=100000, max_depth=11, learning_rate=0.01,random_state=0, subsample=0.8,\n",
    "                                 colsample_bytree=0.6,min_child_weight = 3,reg_alpha=1,reg_lambda = 0.01,n_jobs=-1,tree_method='gpu_hist')\n",
    "    clf.fit(X_train2,y_train2,eval_set = [(X_train2,y_train2),(X_test2,y_test2)], eval_metric = 'auc',early_stopping_rounds=500,verbose=30)\n",
    "    del X_train2,y_train2\n",
    "    temp_predict = clf.predict_proba(X_test2)[:,1]\n",
    "    stack_train[test_index] = temp_predict\n",
    "    y_pred += clf.predict_proba(X_test)[:,1]/5\n",
    "    roc = roc_auc_score(y_test2, temp_predict)\n",
    "    print(roc)\n",
    "    resu1 += roc/5\n",
    "    impor1 += clf.feature_importances_/5\n",
    "    del X_test2,y_test2\n",
    "    gc.collect()\n",
    "print(f'End:{resu1}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "resu = pd.read_csv('../input/ieee-fraud-detection/sample_submission.csv')\n",
    "resu['isFraud'] = y_pred\n",
    "resu.to_csv('xgb.csv',index=False)\n",
    "a= pd.DataFrame()\n",
    "a['train'] = stack_train\n",
    "a.to_csv('xgb_train.csv',index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
